import re
from bs4 import BeautifulSoup
import requests
import pandas as pd

# 设置请求头
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'
}

data = []

# 循环获取每一页的数据
for i in range(10):
    url = f'https://movie.douban.com/top250?start={i * 25}'
    try:
        response = requests.get(url, headers=header)
        response.raise_for_status()  # 如果响应状态码不是 200，则抛出 HTTPError 异常
        soup = BeautifulSoup(response.text, 'html.parser')

        ol = soup.find('ol', class_='grid_view')
        if ol is None:
            continue

        lis = ol.find_all('li')

        for li in lis:
            img_url = li.find('img')['src']

            titles = li.find_all('span', class_='title')
            title = ''.join([t.get_text().strip() for t in titles])

            other_title_element = li.find('span', class_='other')
            other_title = other_title_element.get_text().strip() if other_title_element else ''

            http_url = li.find('a')['href']

            actors = li.find('p', class_='').get_text().strip()

            director_pattern = r"导演: (.*?)主演"
            director_match = re.search(director_pattern, actors, re.S)
            director = director_match.group(1).replace('\xa0', ' ').strip() if director_match else ''

            year = actors.split('\n')[1].strip()

            rating_pattern = r'(\d+)人评价'
            rating_match = re.search(rating_pattern, li.text)
            people_num = rating_match.group(1) if rating_match else ''

            score = li.find('span', class_='rating_num').get_text().strip()

            quote_element = li.find('span', class_='inq')
            quote = quote_element.get_text().strip() if quote_element else ''

            data.append({
                'Image URL': img_url,
                'Title': title,
                'Other Title': other_title,
                'URL': http_url,
                'Director': director,
                'Year': year,
                'People Rated': people_num,
                'Score': score,
                'Quote': quote
            })

    except requests.RequestException as e:
        print(f"Failed to retrieve the page {url}: {e}")
        continue

# 创建 DataFrame
df = pd.DataFrame(data)

# 写入 Excel 文件
df.to_excel('movies_data.xlsx', index=False, engine='openpyxl')


# print(data)
# print(len(data))
# with open ('data.json','w',encoding='utf-8') as f:
#     for i in range(0,len(data)):
#         f.write(str(data[i])+'\n')